package com.stanford.data;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.zip.GZIPInputStream;

import com.stanford.ml.naivebayes.Attribute;
import com.stanford.ml.naivebayes.Instance;
import com.stanford.ml.svm.Feature;

/**
 * A utility class that holds methods to populate user
 * data and access individual data columns
 * 
 * @author Fatih Sunor
 */
public class DataUtil {
	
	public static final int ALL = 0;
	public static final int RATED_ONLY = 1;
	
	private static final String CSV = ".csv";
	private static final String COMMA = ",";
	private static final String CHAR_STANDARD = "US-ASCII";
	private static final int FILE_END = 2;
	private static final long OFFSET = 1262304000000L;

	/**
	 * Builds the list of users in each file
	 * @param zipped file name
	 * @return a map of userlist for each file
	 * @throws IOException if the file does not exist.
	 */
	public static Map<String, List<User>> buildData (String zipFileName) throws IOException{
		List<User> users = new ArrayList<User>();
		InputStream fileStream = new FileInputStream(zipFileName);
		InputStream gzipStream = new GZIPInputStream(fileStream);
		Reader decoder = new InputStreamReader(gzipStream, CHAR_STANDARD);
		BufferedReader buffered = new BufferedReader(decoder);
		String line = null;
		String fileName = null;
		Map<String, List<User>> fileMap = new Hashtable<String, List<User>>();
		while((line = buffered.readLine()) != null){
			List<String> tokens = new ArrayList<String>();
			StringTokenizer tokenizer = new StringTokenizer(line,COMMA);
			while (tokenizer.hasMoreTokens()){
				String next = tokenizer.nextToken();
				if(!next.contains(CSV)){
						tokens.add(next);
				}else{
					tokens.add(next.split(CSV)[1]);
					fileName = next.split(CSV)[0].trim();
					users = new ArrayList<User>();
					fileMap.put(fileName, users);
				}
			}
			if(!(tokens.size()<FILE_END)){
				User user = buildUser(tokens);
				users.add(user);
			}
		}
		return fileMap;
	}
	
	/**
	 * Returns the age column of the data set
	 * @param users is a list of users in the data set
	 * @return an integer array of ages for the selected users
	 */
	public static Integer[] getAges(List<User> users, int filter){
		List<Integer> ages = new ArrayList<Integer>();
		for(User user: users){
			if(filter==RATED_ONLY){
				if(user.getRating() != 0){
					ages.add(user.getAge());
				}
			}else{
				ages.add(user.getAge());
			}
		}
		return (Integer[])ages.toArray(new Integer[ages.size()]);
	}

	/**
	 * Returns the gender column of the data set
	 * @param users is a list of users in the data set
	 * @return an integer array of genders for the selected users
	 */
	public static Integer[] getGenders(List<User> users, int filter){
		List<Integer> genders = new ArrayList<Integer>();
		for(User user: users){
			if(filter==RATED_ONLY){
				if(user.getRating() != 0){
					genders.add(user.getGender());
				}
			}else{
				genders.add(user.getGender());
			}
		}
		return (Integer[])genders.toArray(new Integer[genders.size()]);
	}
	
	/**
	 * Returns the rating column of the data set
	 * @param users is a list of ratings in the data set
	 * @return an integer array of ratings for the selected users
	 */
	public static Integer[] getRatings(List<User> users, int filter){
		List<Integer> ratings = new ArrayList<Integer>();
		for(User user: users){
			if(filter==RATED_ONLY){
				if(user.getRating() != 0){
					ratings.add(user.getRating());
				}
			}else{
				ratings.add(user.getRating());
			}
		}
		return (Integer[])ratings.toArray(new Integer[ratings.size()]);
	}
	
	/**
	 * Builds the user object based on a list of string tokens
	 * @param tokens compose the details of a user
	 * @return a single user
	 */
	private static User buildUser(List<String> tokens) {
		String token = tokens.get(0);
		int rating = Integer.valueOf(token.substring(token.length()-1));
		if(tokens.get(0).startsWith("-")){
			rating = -rating;
		}
		Date timeStamp = new Date(Long.valueOf(tokens.get(1))+OFFSET);
		int age = Integer.valueOf(tokens.get(2));
		int gender = Integer.valueOf(tokens.get(3));
		String city = tokens.get(4);
		String state = tokens.get(5);
		String country = tokens.get(6);
		User user = new User(rating, timeStamp, age, gender, city, state, country);
		for(int i = 7; i<tokens.size(); i++){
			String[] tagMap = tokens.get(i).split(":");
			user.appendTag(tagMap[0], Integer.valueOf(tagMap[1]));
		}
		return user;
	}
	
	/**
	 * Builds the feature table where columns represent each feature
	 * and rows represent the feature values for each instance
	 * @param features is a list of integer arrays representing the values
	 * @return Feature[][] table
	 */
	public static Feature[][] getFeatures(List<Integer[]> features) {
		Feature[][] featureTable = new Feature[features.get(0).length][features.size()];
		for(int i=0;i<features.get(0).length;i++){
			for(int j=0; j<features.size(); j++){
				featureTable[i][j] = new Feature(j, features.get(j)[i]);
			}
		}
		return featureTable;
	}

	/**
	 * A utility method that converts instance lists to instance arrays
	 * @param list of instances
	 * @return an array of instances
	 */
	public static Instance[] instanceListToArray(List<User> list) {
		Instance[] instances = new Instance[list.size()];
		int count = 0;
		for(User user: list){
			instances[count] = user;
			count++;
		}
		return instances;
	}
	
	/**
	 * 
	 * @param list
	 * @return
	 */
	public static Attribute[] attributeListToArray(List<Attribute> list) {
		Attribute[] attributes = new Attribute[list.size()];
		int count = 0;
		for(Attribute attribute: list){
			attributes[count] = attribute;
			count++;
		}
		return attributes;
	}

	/**
	 * Utility method that 
	 * @param list of users 
	 * @return list of users who rated
	 */
	public static List<User> filterRatedUsers(List<User> list) {
		List<User> users = new ArrayList<User>();
		for(User user: list){
			if(user.getRating() != 0){
				users.add(user);
			}
		}
		return users;
	}

	/**
	 * Returns an array of cities
	 * @param users list of users
	 * @param filter whether rated users should be filtered
	 * @return an array of countries
	 */
	public static String[] getCities(List<User> users, int filter) {
		List<String> cities = new ArrayList<String>();
		for(User user: users){
			if(filter==RATED_ONLY){
				if(user.getRating() != 0){
					cities.add(user.getCity());
				}
			}else{
				cities.add(user.getCity());
			}
		}
		return (String[])cities.toArray(new String[cities.size()]);
	}

	/**
	 * Returns an array of states
	 * @param users list of users
	 * @param filter whether rated users should be filtered
	 * @return an array of countries
	 */
	public static String[] getStates(List<User> users, int filter) {
		List<String> states = new ArrayList<String>();
		for(User user: users){
			if(filter==RATED_ONLY){
				if(user.getRating() != 0){
					states.add(user.getState());
				}
			}else{
				states.add(user.getState());
			}
		}
		return (String[])states.toArray(new String[states.size()]);
	}

	/**
	 * Returns an array of countries
	 * @param users list of users
	 * @param filter whether rated users should be filtered
	 * @return an array of countries
	 */
	public static String[] getCountries(List<User> users, int filter) {
		List<String> countries = new ArrayList<String>();
		for(User user: users){
			if(filter==RATED_ONLY){
				if(user.getRating() != 0){
					countries.add(user.getCountry());
				}
			}else{
				countries.add(user.getCountry());
			}
		}
		return (String[])countries.toArray(new String[countries.size()]);
	}

	/**
	 * Utility to revert the position of the key and value
	 * @param tokenCount is a map containing counts of tokens
	 * @return a map with values reverted with key
	 */
	public static Map<Integer, String> revertStringKeyWithIntegerValue(Map<String, Integer> tokenCount) {
		Map<Integer, String> reversed = new HashMap<Integer, String>();
		for(String token: tokenCount.keySet()){
			if(reversed.containsKey(tokenCount.get(token))){
				reversed.put(tokenCount.get(token), reversed.get(tokenCount.get(token))+", "+token);
			}else{
				reversed.put(tokenCount.get(token), token);
			}
		}
		return reversed;
	}

	/**
	 * Returns all factors for classification
	 * @param users is a list of users
	 * @return a map of factors and data for those factors
	 */
	public static Map<String, Integer[]> getAllFactors(List<User> users) {
		 Map<String, Integer[]> factors = new HashMap<String, Integer[]>();
		 factors.put("Age", getAges(users, RATED_ONLY));
		 factors.put("Gender", getGenders(users, RATED_ONLY));
		 factors.putAll(getTags(users,RATED_ONLY));
		 return factors;
	}

	private static Map<String, Integer[]> getTags(List<User> users, int filter) {
		Map<String, Integer[]> tagMap = new HashMap<String, Integer[]>();
		Set<String> allTags = getAllTags(users);
		for(String tag: allTags){
			List<Integer> count = new ArrayList<Integer>();
			for(User user: users){
				if(filter==RATED_ONLY){
					if(user.getRating() != 0){
						if(user.getTags().get(tag)!=null){
							count.add(user.getTags().get(tag));
						}else{
							count.add(0);
						}
					}
				}else{
					if(user.getTags().get(tag)!=null){
						count.add(user.getTags().get(tag));
					}
				}
			}
			tagMap.put(tag,(Integer[])count.toArray(new Integer[count.size()]));
		}
		return tagMap;
	}

	private static Set<String> getAllTags(List<User> users) {
		Set<String> allTags = new HashSet<String>();
		for(User user: users){
			allTags.addAll(user.getTags().keySet());
		}
		return allTags;
	}	
}
